package fetch;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;


import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.impl.client.DefaultHttpClient;


import com.ibm.json.java.JSONArray;
import com.ibm.json.java.JSONObject;

/**
 *  http://bskk.com/forum.php?mod=viewthread&tid=2826303&moblie=no
	http://bskk.com/forum.php?mod=viewthread&tid=2826303&extra=&page=2&moblie=no
	http://bskk.com/forum.php?mod=viewthread&tid=2826303&extra=&page=2
 * @author Administrator
 *
 */

public class FetchBskkPage {
	
	//private static Logger logger = Logger.getLogger(FetchBskkPage.class);

	public static int count = 0 ;//记录有效帖子数目
	public static int loopcount = 0 ;
	public static int tid = 0;
	
	/**
	 * @param args
	 */
	public static void main(String[] args) 
	{
		//PropertyConfigurator.configure ("./config/log4j.properties");
		System.out.println("<----  " +  "Version : 130318"  + "  ---->");
		
		SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
		System.out.println("<----  " +  "start time: " + df.format(new Date())  + "  ---->");
		
		
		//logger.debug("Start Fetch Page : " + 136150 );
		
		if(args == null || args.length < 2)
		{
			System.out.println("Please input start pageid and end pageid !");
			
			return ;
		}
		
		int startpageid = Integer.parseInt(args[0]);
		
		int endpageid = Integer.parseInt(args[1]);
		
		
		System.out.println("<----  " +  "Start Fetch Page : " + startpageid   + "  ---->");
		//startpageid = 136150    endstartpageid = 2827929
		try
		{
			for(int i = startpageid ; i <= endpageid ; i ++ )
			{
				loopcount ++;
				tid = i;
				saveJSONPageFile(getJSONBskkPage(i + ""));
				//getJSONBskkPage(i + "");
				
				if (i%100 ==0)
				{
					System.out.println("<----  " + "Current time : " + df.format(new Date()) + "  Current tid : " + i + "  Current effective Posts ： " + count 
							+ "   Curretn loopcount：" + loopcount + "  ---->");
				}
					
			}
			
		}catch(Exception e)
		{
			e.printStackTrace();
			
		}finally
		{
			System.out.println("<----  " +  "循环次数：" + loopcount  + "  ---->");
			System.out.println("<----  " + "帖子ID: " + tid   + "  ---->");
			System.out.println("<----  " +"有效总帖子数目： " + count   + "  ---->");
			System.out.println("<----  " + "End Fetch Page " + endpageid  + "  ---->");
			System.out.println("<----  " + "end time: " + df.format(new Date()) + "  ---->");
		}
		
	
		/**
		logger.debug("循环次数：" + loopcount);
		logger.debug("帖子ID: " + tid);
		logger.debug("总帖子数目： " + count);
		logger.debug("End Fetch Page " + 137150);
		*/
		
		//saveJSONPageFile(getJSONBskkPage("2826303"));
		
		//readJSONPageFile("./file/log.txt");
		
		
		
	}
	
	
	public static void readJSONPageFile(String fileName)
	{
		 File file = new File(fileName);
		 
		 BufferedReader reader = null;
		 
		 try
		 {
			  reader = new BufferedReader(new FileReader(file));
			 
			  String tempString = null;
			  
			  int line = 1;
			  
			  while ((tempString = reader.readLine()) != null)
			  {
			      //显示行号
			      //System.out.println("line "+ line + ": " + tempString);
			      
			      JSONObject jo = JSONObject.parse(tempString);
			      System.out.println(jo.get("surl"));
			      JSONArray jpagings = JSONArray.parse(jo.get("body").toString());
			      
			      System.out.println("jpagings size : " + jpagings.size());
			      for(int i = 0 ; i < jpagings.size(); i++)
			      {
			    	  JSONObject paging = (JSONObject) jpagings.get(i);
			    	//  System.out.println(paging);
			    	  
			    	  int pagenum = i +1;
			    	  System.out.println("pagenum : " + pagenum);
			    	  System.out.println(paging.get("page" + pagenum));
			      }
			      
			    
			     
			    		  
			  }
		 }catch (IOException e) 
		 {
		     e.printStackTrace();
		 } finally 
		 {
		     if (reader != null) 
		     {
		      try 
		      {
		       reader.close();
		      } catch (IOException e1) 
		      {
		      }
		     }
		    }
	}
	
	
	
	public static void saveJSONPageFile(JSONObject jpage)
	{
		 BufferedWriter writer = null;
		 
		 String filePath = "./file/pages.txt";
		 
		try
		{
			if(jpage == null)
				return ;
			
			writer =  new BufferedWriter(new FileWriter(new File(filePath),true));
			
			writer.write(jpage.toString());
			
			writer.newLine();
			
			//writer.write("");
			writer.flush();
			
			writer.close();
		}catch(Exception e)
		{
			System.out.println("<----  " +  "Error page url: " + jpage.get("surl")  + "  ---->");
			//logger.debug("Error page url: " + jpage.get("surl"));
			
			e.printStackTrace();
			
		}
		
		count ++ ;
		System.out.println("<----  " +  "Effective page url : " + jpage.get("surl")   + "  ---->");
		
		
		
	}
	
	public static JSONObject getJSONBskkPage(String uid)
	{
		String sUrl = "";
		
		HttpClient httpclient = new DefaultHttpClient();
		
		JSONArray pagings = new JSONArray();
		
		JSONObject jpage = new JSONObject();
		
		boolean flag = false;
		
		try
		{
			sUrl = "http://bskk.com/forum.php?mod=viewthread&tid=" + uid ;
			
			System.out.println("current url: " + sUrl);
			
			for(int i = 1 ; i < 100 ; i ++)
			{
				JSONObject paging = new JSONObject();
				// Create an instance of HttpGet.
				HttpGet httpget = new HttpGet(sUrl + "&extra=&page=" + i);
							
				// Create a response handler
				ResponseHandler<String> responsehandler = new BasicResponseHandler();
							
				String responseBody = httpclient.execute(httpget,responsehandler);
				
				
				if(responseBody.indexOf("下一页")<0)
				{
					paging.put("page"+ i, responseBody);
					pagings.add(paging);
					break;
				}
				
				paging.put("page"+ i, responseBody);
				
				pagings.add(paging);
				
				
			}
			
			
		}catch(Exception e)
		{
			System.out.println("<----  " +  "Error page url: " + sUrl   + "  ---->");
			//logger.debug("Error page url: " + sUrl);
			
			e.printStackTrace();
			
		}finally
		{
			httpclient.getConnectionManager().shutdown();
		}
		
		
		jpage.put("surl", sUrl);
		
		jpage.put("body", pagings);
		
		if(!isIncludeKeyWord(pagings,"日日慈"))//过滤其他帖子，只保留日日慈帖子
			return null;
		
		//System.out.println("page num : " + pagings.size());
		return jpage;
	}
	
	
	public static boolean isIncludeKeyWord(JSONArray pagings, String keyword)
	{
		boolean flag = false;
		
		if (pagings == null || pagings.size() == 0)
				return false;
		for(int i = 0 ; i < pagings.size() ; i++)
		{
			 int pagenum = i +1;
			 
			 JSONObject paging = (JSONObject) pagings.get(i);
			 
			if(paging.get("page" + pagenum).toString().indexOf(keyword) >= 0)
			{
				flag = true;
				
				break;
			}
		}
		
		return flag;
	}

	
	
	public static void viewHashMap(HashMap map)
	{
		Iterator iter = map.entrySet().iterator();
		
		while(iter.hasNext())
		{
			Map.Entry entry = (Entry) iter.next();
			
			System.out.println(entry);
		}
	}
}
